package cn.haut.online_read.crawler.sanwenzx;

import cn.haut.online_read.utils.toMapList;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Arrays;
import java.util.List;
import java.util.Map;

import static cn.haut.online_read.crawler.sanwenzx.constant.Constant.BASE_URL;

/**
 * @author xing
 * @date 2021-05-23 0:10
 */
public class MajorContentCrawler implements PageProcessor {

    private Site site = Site.me().setSleepTime(1);

    @Override
    public void process(Page page) {
        List<String> imgSrcList = page.getHtml().xpath("//ul[@class='e2']/li/a[1]/img/@src").all();
        for (int i = 0; i < imgSrcList.size(); i++) {
            imgSrcList.set(i, BASE_URL + imgSrcList.get(i));
        }

        List<String> titleList = page.getHtml().xpath("//ul[@class='e2']/li/a[2]/b/text()").all();
        List<String> descList = page.getHtml().xpath("//ul[@class='e2']/li/p/text()").all();
        for (int i = 0; i < descList.size(); i++) {
            descList.set(i, descList.get(i).trim());
        }

        List<String> urlList = page.getHtml().xpath("//ul[@class='e2']/li/a[1]/@href").all();

        List<Map<String, Object>> maps = toMapList.get(
                Arrays.asList("imgUrl", "name", "desc", "url"),
                imgSrcList, titleList, descList, urlList
        );

        page.putField("articles", maps);

        String total = page.getHtml().xpath("//span[@class='pageinfo']/strong[2]/text()").get();
        String pageBaseUrl = page.getHtml().xpath("//ul[@class='pagelist']/li/a/@href").get().replaceAll("[0-9].html", "");
        page.putField("total", Integer.parseInt(total));
        page.putField("pageBaseUrl", pageBaseUrl);
    }

    @Override
    public Site getSite() {
        return site;
    }
}
